#encoding=utf8
import urllib2
import md5

_user_agent_ = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 1.1.4322)"



def fetch(url, read_cache=True):
  key = md5.new(url).hexdigest()
  path = "tmp/" + key
  if read_cache:
    try:
      fin = open(path, "r")
      content = fin.read()
      fin.close()
      return content
    except:
      pass

  try:
    connection = urllib2.urlopen(urllib2.Request(url,headers={"user-agent": _user_agent_}))
    content = connection.read()
    try:
      connection.close()
    except: pass

    fin = open(path, "w")
    fin.write(content)
    fin.close()
    return content
  except:
    print "failed to fetch %s" % url
    return None